package tools;

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Serializable;
import java.util.ArrayList;


/**
 * Classe de racinisation (stemming) des mots
 * en francais.
 * Modification l�g�re du package SnowBall
 * http://snowball.tartarus.org/download.php
 * @author xtannier
 *
 */
public class FrenchStemmer extends org.tartarus.snowball.ext.frenchStemmer implements Normalizer, Serializable {

	/**
	 * 
	 */
	private static final long serialVersionUID = 1L;

	private static short REPEAT = 1;
	
	private ArrayList<String> stopWords;
	
	public FrenchStemmer() {
		this.stopWords = new ArrayList<String>();
	}
	
	public FrenchStemmer(String stopWordFileName) throws IOException {
		this.stopWords = new ArrayList<String>();
		
		//lecture du fichier texte	
		InputStream ips=new FileInputStream(stopWordFileName); 
		InputStreamReader ipsr=new InputStreamReader(ips);
		BufferedReader br=new BufferedReader(ipsr);
		String line;
		while ((line=br.readLine())!=null){
			this.stopWords.add(line);
		}
		br.close();
	}


	public ArrayList<String> normalize(String fileName) throws IOException {		
		String text = "";
		//lecture du fichier texte	
		InputStream ips=new FileInputStream(fileName); 
		InputStreamReader ipsr=new InputStreamReader(ips);
		BufferedReader br=new BufferedReader(ipsr);
		String line;
		while ((line=br.readLine())!=null){
			text += line + " ";
		}
		br.close(); 
		
		ArrayList<String> words = (new FrenchTokenizer()).tokenize(text.toLowerCase());
		ArrayList<String> result = new ArrayList<String>();
		for (String word : words) {
			// on ajoute le mot dans la liste s'il n'appartient pas � la liste des mots-cl�s.
			// Id�alement il faudrait utiliser une structure de donnees plus efficace que la liste,
			// mais ce n'est pas le sujet.
			this.setCurrent(word);
			for (int i = REPEAT; i != 0; i--) {
				this.stem();
			}
			result.add(this.getCurrent());
		}
		return result;
	}

	public static void main(String[] args) {
		String stopWordFileName = "/home/wchen/Study/EIT/frenchST.txt";
		String word = "/home/wchen/Study/EIT/lemonde/texte.95-1.txt";
		try {
			Normalizer stemmer = new FrenchStemmer(stopWordFileName);
			String encoding = "ISO-8859-1";
			ArrayList<String> resultat = stemmer.normalize(word,encoding,true);
			
			
			for(String test:resultat){
				System.out.println(test);
			}
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}

	@Override
	public ArrayList<String> normalize(String fileName, String encoding, boolean removeStopWords)
			throws IOException {
		
		if(removeStopWords){
			String text = "";
			//lecture du fichier texte	
			InputStream ips=new FileInputStream(fileName); 
			InputStreamReader ipsr=new InputStreamReader(ips,encoding);
			BufferedReader br=new BufferedReader(ipsr);
			String line;
			while ((line=br.readLine())!=null){
				text += line + " ";
			}
			br.close(); 
			
			ArrayList<String> words = (new FrenchTokenizer()).tokenize(text.toLowerCase());
			ArrayList<String> result = new ArrayList<String>();
			for (String word : words) {
				
					boolean remove = false;
					for(String stopWord : this.stopWords){
						if(word.equals(stopWord)){
							remove = true;
						}
					}
					if(!remove){
					
						this.setCurrent(word);
						for (int i = REPEAT; i != 0; i--) {
							this.stem();
						}
						result.add(this.getCurrent());
					}
			}
			return result;
		}
		else{
			return this.normalize(fileName);
		}
	}

	@Override
	public ArrayList<String> normalizeText(String text, boolean removeStopWords) {
		ArrayList<String> words = (new FrenchTokenizer()).tokenize(text.toLowerCase());
		ArrayList<String> result = new ArrayList<String>();
		
		if(removeStopWords){	
			for (String word : words) {
				boolean remove = false;
				for(String stopWord : this.stopWords){
					if(word.equals(stopWord)){
						remove = true;
					}
				}
				if(!remove){
					
					this.setCurrent(word);
					for (int i = REPEAT; i != 0; i--) {
						this.stem();
					}
					result.add(this.getCurrent());
				}
			}
			return result;
		}
		else{
			for (String word : words) {
				// on ajoute le mot dans la liste s'il n'appartient pas a la liste des mots-cles.
				// Idealement il faudrait utiliser une structure de donn�es plus efficace que la liste,
				// mais ce n'est pas le sujet.
				this.setCurrent(word);
				for (int i = REPEAT; i != 0; i--) {
					this.stem();
				}
				result.add(this.getCurrent());
			}
			return result;
		}
	}
}
